'use strict'
const cnblogs = require('../models/cnblog');
var i = 1;
var Crawler = require("crawler");

module.exports = () => {
    var c = new Crawler({
        rateLimit: 10,
        maxConnections: 1,
        // This will be called for each crawled page
        callback: function (error, res, done) {
            if (error) {
                console.log(error);
            } else {
                var $ = res.$;
                var post = $(" #wrapper #main #post_list .post_item").toArray();
                post.forEach(element => {
                    var reg = /\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}/;
                    var title = $(element).find(' .post_item_body h3 a').text();
                    var author = $(element).find(' .post_item_body .post_item_foot .lightblue').text();
                    var newstime = reg.exec($(element).find(' .post_item_body .post_item_foot ').text()).toString();
                    cnblogs.create({ title: title, author: author, newstime: newstime });
                    //console.log(author,newstime);
                });
                // $ is Cheerio by default
                //a lean implementation of core jQuery designed specifically for the server
                //console.log($("title").text());
            }
            done();
        }
    });

    // Queue just one URL, with default callback
    //c.queue('https://www.cnblogs.com/');
    while (i <= 200) {
        c.queue('https://www.cnblogs.com/sitehome/p/' + i);
        i = i + 1;
    }

}